import sys


def dealWithQueryLevelFeature(inputFileName1):
    print "dealWithQueryLevelFeature*******************"
    return_query_Dict = {}
    inputFileHandler1 = open(inputFileName1, "r")
    #keyWordsDict = {}
    
    for currentLine in inputFileHandler1.readlines():
        currentLineElements = currentLine.split("\t")
        queryNumber = currentLineElements[0].strip()
        adhocTopicNumber = currentLineElements[2].strip()
        relevenceScore = currentLineElements[4].strip()
        queryInfoString = queryNumber + " " + adhocTopicNumber + " " + relevenceScore + " "
        
        queryContent = currentLineElements[1].strip()
        warcTrecID = currentLineElements[3].strip()
        
        returnTurple = (queryContent, warcTrecID)
        if returnTurple not in return_query_Dict:
            return_query_Dict[returnTurple] = queryInfoString
        else:
            print "Error1"
            print returnTurple
            sys.exit(1)
            
    
    return return_query_Dict
    #The key is the thing we hunt for.
      
    '''   
        for currentTerm in queryContentElement:
            key = currentTerm + "_" + warcTrecID
            if key not in keyWordsDict:
                keyWordsDict[key] = 1
            else:
                keyWordsDict[key] += 1
    
    keyWordsDict
    sorted_keyWordsDict = sorted(keyWordsDict.iteritems(), key=operator.itemgetter(0))
    
    print len(keyWordsDict)
    print type(sorted_keyWordsDict)
    
    
    for currentElement in sorted_keyWordsDict:
        (queryTerm_warcID,freq) = currentElement
        print queryTerm_warcID
        #print "freq:",freq
    '''
    inputFileHandler1.close()


def getDocumentLevelInfo():
    outputFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/finalTrainingFiles/feature_file_machine_generated.arff"
    
    outputQueryFeatureFileHandler = open(outputFeatureFileName,"w")
    
    outputQueryFeatureFileHandler.write("@relation ranking" + "\n")
    outputQueryFeatureFileHandler.write("@attribute doc_id numeric" + "\n")
    outputQueryFeatureFileHandler.write("@attribute doc_words numeric" + "\n")
    outputQueryFeatureFileHandler.write("@attribute doc_distinct_words numeric" + "\n")
    outputQueryFeatureFileHandler.write("@attribute doc_size numeric" + "\n")
    outputQueryFeatureFileHandler.write("@attribute doc_waterloo_spam real" + "\n")
    outputQueryFeatureFileHandler.write("@attribute url_length numeric" + "\n")
    outputQueryFeatureFileHandler.write("@attribute url_in_top_million {1, 0}" + "\n")
    outputQueryFeatureFileHandler.write("@data" + "\n")
    
    
    #outputQueryFeatureFileHandler.write("0,2442,798,18379,63,52,1" + "\n")
    
    print "There are 19381 lines in the file: query_features_2011_adhoc.txt"
    
    print "Get documents which needed to be evaluated."
    inputFileForUseEvaluatedDocumentsInfoFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_features_2011_adhoc.txt"
    inputFileForUseEvaluatedDocumentsInfoFileHandler = open(inputFileForUseEvaluatedDocumentsInfoFileName, "r")
    finalDocumentDict = {}
    
    for currentLine in inputFileForUseEvaluatedDocumentsInfoFileHandler.readlines():
        currentLineElements = currentLine.split("\t")
        WARCTRECID = currentLineElements[3]
        if WARCTRECID not in finalDocumentDict:
            finalDocumentDict[WARCTRECID] = 1
        else:
            finalDocumentDict[WARCTRECID] += 1
    inputFileForUseEvaluatedDocumentsInfoFileHandler.close()
    
    print "# of documents needed to be collected:",len(finalDocumentDict)
    
    '''
    sum = 0
    for eachElement in finalDocumentDict:
        sum += finalDocumentDict[eachElement]
    print sum
    '''
    
    print "Load document_features.txt and add the corresponding document level features"
    testDict = {}
    outputCounter = 0
    inputDocumentFeatureFileName = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/document_features.txt"
    inputDocumentFeatureFileHandler = open(inputDocumentFeatureFileName, "r")
    #print len(inputDocumentFeatureFileHandler.readlines())
    NOTRecordedCounter = 0
    for eachInputLine in inputDocumentFeatureFileHandler.readlines():
        eachInputLineElements = eachInputLine.strip().split(" ")
        if len(eachInputLineElements) != 9:
            NOTRecordedCounter += 1
            #print "sth. wrong"
            #print eachInputLine
            #print eachInputLineElements
            #sys.exit(1)
        else:
            WARCTRECID = eachInputLineElements[0]
            if WARCTRECID not in testDict:
                testDict[WARCTRECID] = 1
            else:
                testDict[WARCTRECID] += 1
            
            if WARCTRECID not in finalDocumentDict:
                pass
            else:
                outputCounter += 1
                URLContent = eachInputLineElements[1]
                docID = eachInputLineElements[2]
                hamProbability = eachInputLineElements[3]
                isPopular = eachInputLineElements[4]
                URLLength = eachInputLineElements[5]
                docSizeInBytes = eachInputLineElements[6]
                docWords = eachInputLineElements[7]
                docDistantWords = eachInputLineElements[8]
                
                outputQueryFeatureFileHandler.write(docID + "," + docWords +  "," + docDistantWords + ","+ docSizeInBytes + ","+ hamProbability + "," + URLLength+ "," + isPopular +"\n")
          
            
    outputQueryFeatureFileHandler.close()
    print "len(testDict):",len(testDict)
    '''
    sum = 0
    for eachElement in testDict:
        sum += testDict[eachElement]
    print sum
    '''
    print "# of documents written:",outputCounter
    print "# of documents NOT recorded(Cause the format is not right):",NOTRecordedCounter
    print "Job Done!"
    


'''
inputFileName1 = "/data1/team/weijiang/machine-learning-project-related/featuresGenerated/query_features_2011_adhoc.txt"
dict1 = dealWithQueryLevelFeature(inputFileName1)
print len(dict1)
for queryWARCTRECIDTupleElement in dict1:
    print queryWARCTRECIDTupleElement,dict1[queryWARCTRECIDTupleElement]
'''
getDocumentLevelInfo()    
    
    
    
    
    
    
